{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "6534771f",
   "metadata": {},
   "source": [
    "# BERT embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "cbec9d57",
   "metadata": {},
   "outputs": [],
   "source": [
    "%run -i \"../util/util_simple_classifier.ipynb\"\n",
    "%run -i \"../util/file_utils.ipynb\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "9cf322f5",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sentence_transformers import SentenceTransformer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "213a820d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_sentence_vector(text, model):\n",
    "    sentence_embeddings = model.encode([text])\n",
    "    return sentence_embeddings[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "07b2c0e7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 2.94217980e-03 -7.93536603e-02 -2.82228496e-02 -5.13779782e-02\n",
      "  -6.44981042e-02  9.83557850e-02  1.09671958e-01 -3.26390602e-02\n",
      "   4.96566631e-02  2.56580133e-02 -1.08482063e-01  1.88441798e-02\n",
      "   2.70963665e-02 -3.80690470e-02  2.42502335e-02 -3.65605950e-03\n",
      "   1.29364491e-01  4.32255343e-02 -6.64561391e-02 -6.93060979e-02\n",
      "  -1.39410645e-01  4.36719768e-02 -7.85463024e-03  1.68625098e-02\n",
      "  -1.01160072e-02  1.07926019e-02 -1.05814040e-02  2.57284809e-02\n",
      "  -1.51516097e-02 -4.53920700e-02  7.12087378e-03  1.17573030e-01\n",
      "  -1.31499115e-03 -2.59251408e-02 -8.34854692e-02  3.65455374e-02\n",
      "  -2.86979042e-02  3.15047689e-02  3.97393666e-02  5.78806922e-02\n",
      "  -2.24544741e-02  7.98929557e-02  4.24228190e-03 -1.34443259e-02\n",
      "  -7.81929493e-02 -1.82036217e-02  6.06966671e-03 -3.64662483e-02\n",
      "   8.80980566e-02  5.81041686e-02 -3.77041218e-03  2.63834409e-02\n",
      "   4.09417860e-02  5.37395552e-02 -1.32099977e-02  2.30074134e-02\n",
      "  -5.78504801e-02  9.60877612e-02  1.13228159e-02 -4.31320667e-02\n",
      "   7.90961273e-03  1.29869254e-02 -2.28543654e-02 -2.93108057e-02\n",
      "   7.18075922e-03  4.32306677e-02  3.09592448e-02  3.83532909e-03\n",
      "   1.96842086e-02 -4.86251190e-02 -1.20897833e-02  4.83374149e-02\n",
      "  -7.31307864e-02 -2.21118126e-02  1.67823881e-02  9.25477669e-02\n",
      "   1.57828480e-02  1.92148760e-02  1.24158757e-02 -5.50739355e-02\n",
      "   7.69023672e-02 -3.22304405e-02  3.08283884e-02 -1.12228133e-01\n",
      "  -5.36542721e-02 -3.33914869e-02 -6.65069818e-02  8.25061575e-02\n",
      "  -2.03155633e-02  2.28298968e-03 -3.63824777e-02  1.10852392e-02\n",
      "   1.14273354e-02 -3.20380665e-02 -1.39946342e-02 -3.42153199e-02\n",
      "  -2.37518698e-02 -3.35111544e-02 -3.83234136e-02  1.21933706e-01\n",
      "   2.61066873e-02  3.73761132e-02  4.71137725e-02  4.55218293e-02\n",
      "  -2.62210779e-02  3.29928729e-03  1.11120567e-02  9.34213102e-02\n",
      "   8.71658977e-03 -3.22441272e-02  6.66072890e-02  7.82261882e-03\n",
      "   3.51459906e-02 -9.30157024e-03  2.05826014e-02 -1.55156916e-02\n",
      "   6.70894608e-02  7.46397525e-02  6.74724132e-02  1.93882063e-02\n",
      "   3.69183198e-02  5.56740463e-02 -3.96636464e-02 -1.64558962e-02\n",
      "  -9.56231803e-02 -8.36475100e-03 -5.01270257e-02 -4.67768140e-33\n",
      "  -2.33913232e-02 -1.37979193e-02  8.22425857e-02 -3.51544144e-03\n",
      "   2.00477298e-02  1.59348808e-02 -6.48919120e-02  4.59935740e-02\n",
      "  -3.58517207e-02  1.47760957e-02  1.04296999e-03  6.40693977e-02\n",
      "   2.32883310e-03 -2.10527051e-02  3.91378589e-02  1.56031661e-02\n",
      "   1.93105210e-02 -3.54503542e-02 -6.28079986e-03 -2.26757694e-02\n",
      "  -8.12372938e-02  6.62341416e-02 -4.34660091e-04  5.56828119e-02\n",
      "  -2.11597346e-02 -8.98617506e-03  7.15201497e-02 -5.76146133e-02\n",
      "  -6.52829856e-02  1.93028972e-02  3.47890370e-02 -4.12579179e-02\n",
      "  -4.47054021e-03  1.54107325e-02  6.74647791e-03 -3.86915728e-02\n",
      "  -4.05457653e-02  2.49731578e-02 -2.12479308e-02 -4.08775546e-02\n",
      "   3.26235518e-02 -4.60083038e-02 -8.82487223e-02  1.21176116e-01\n",
      "   2.61267107e-02  3.99443693e-02  4.70666820e-03 -4.43025194e-02\n",
      "   2.94008516e-02 -3.82606201e-02  1.05659626e-02  3.45226079e-02\n",
      "  -2.02615280e-02  5.67788295e-02  3.62922624e-02 -6.45636965e-06\n",
      "   4.20921445e-02  5.28537333e-02 -6.36702329e-02  3.32096294e-02\n",
      "  -4.44335444e-03  6.35157600e-02  1.49307307e-03  5.73670445e-03\n",
      "   2.48714676e-03  4.02897410e-02  1.73509885e-02  5.09857386e-02\n",
      "   6.41008373e-03 -7.72566497e-02 -4.54469211e-02 -6.32283464e-03\n",
      "   1.91583578e-02  3.19066755e-02  5.23454184e-03 -2.89178221e-03\n",
      "   5.21416776e-02 -1.00331932e-01  2.69881338e-02 -6.56073540e-02\n",
      "  -4.62833308e-02 -1.04512058e-01  4.65779845e-03 -1.19208796e-02\n",
      "  -2.84514539e-02 -3.93571099e-03  1.08878808e-02 -5.87290190e-02\n",
      "  -1.95604824e-02 -6.36863634e-02 -1.47502631e-01  1.59961637e-02\n",
      "   9.31285620e-02  6.86270222e-02 -1.78965330e-02  3.28041462e-33\n",
      "   1.19608315e-02 -3.43739092e-02  1.55591974e-02  1.86485890e-02\n",
      "   5.34957275e-03 -1.31002832e-02 -3.19519639e-02  7.52187753e-03\n",
      "  -2.04244591e-02  6.65009916e-02  2.39214487e-02 -7.76461586e-02\n",
      "   1.30739622e-02  2.52394434e-02 -3.41380015e-02  1.31033417e-02\n",
      "  -2.64725331e-02  1.29369656e-02  1.80937089e-02  1.54408719e-02\n",
      "  -5.50816916e-02  2.18439288e-02 -3.38429026e-02  5.43976463e-02\n",
      "  -6.37546778e-02  5.33061177e-02 -4.55683954e-02  3.90717462e-02\n",
      "  -1.66391004e-02  1.47519276e-01  1.09325938e-01 -1.05439555e-02\n",
      "   7.67368404e-03 -1.18745625e-01 -1.56953577e-02  3.47008482e-02\n",
      "   4.65803705e-02 -1.32938828e-02 -5.09740487e-02 -1.78620461e-02\n",
      "  -1.11284010e-01 -4.09153895e-03  2.36834045e-02  9.51901376e-02\n",
      "   5.17866723e-02  6.52891845e-02  4.90683243e-02  4.12996113e-02\n",
      "  -1.11020029e-01 -2.28067730e-02 -7.26590976e-02  1.78629812e-02\n",
      "  -2.83320937e-02 -1.45523595e-02 -2.72194017e-02  4.40445840e-02\n",
      "   6.50395676e-02 -7.35710887e-03 -4.09941375e-02 -4.05894369e-02\n",
      "  -3.54929604e-02  2.90991273e-02  2.28240006e-02  3.39522795e-03\n",
      "   4.69186231e-02  1.22833271e-02  1.16244167e-01 -2.47269962e-02\n",
      "  -3.29611860e-02 -3.95420976e-02 -1.85898542e-02 -3.09413075e-02\n",
      "  -6.62771687e-02  5.51542789e-02 -3.55085209e-02 -2.07906943e-02\n",
      "   4.06854935e-02  2.22931542e-02 -1.34954611e-02  5.12118153e-02\n",
      "   2.22113114e-02 -1.01478817e-02 -6.56570569e-02  1.20399324e-02\n",
      "   1.60541497e-02  3.43546048e-02  2.00314540e-02 -1.55158695e-02\n",
      "  -6.88960776e-02  5.53822964e-02  5.26592620e-02  2.50734133e-03\n",
      "  -5.82410097e-02 -9.05284844e-03  1.83004197e-02 -1.22338406e-08\n",
      "   7.80071467e-02  3.06937229e-02 -2.10351497e-02 -3.21454778e-02\n",
      "  -1.70106515e-02 -3.25915255e-02  2.38911584e-02 -7.62749556e-03\n",
      "  -1.03250206e-01 -4.22040820e-02 -2.68117841e-02 -4.96933311e-02\n",
      "   1.00441240e-02  7.71872550e-02  8.81059468e-02  7.53305759e-03\n",
      "   4.95146513e-02 -5.38602732e-02 -2.17240229e-02  3.76712419e-02\n",
      "   8.88131745e-03 -7.09614530e-03  1.52336359e-01 -7.94296041e-02\n",
      "  -4.91029732e-02 -1.06822774e-01 -3.00454088e-02 -8.15807804e-02\n",
      "   4.53118309e-02  3.39330807e-02 -5.16491570e-03  8.15858096e-02\n",
      "  -9.80188884e-03 -3.40925306e-02 -4.92097028e-02 -5.39324135e-02\n",
      "   1.13763206e-01 -9.35967937e-02  2.55571343e-02 -5.30686341e-02\n",
      "  -8.06598514e-02  7.62472153e-02 -6.53353799e-03 -7.57136568e-02\n",
      "  -5.17365448e-02 -8.21988434e-02  1.57644212e-01 -2.12260000e-02\n",
      "  -3.38822082e-02 -1.84449386e-02 -2.76150536e-02  6.25501648e-02\n",
      "   9.38665215e-03  6.75280765e-02  8.42629522e-02  1.54313107e-03\n",
      "  -1.07518718e-01  6.12090006e-02  7.30485131e-04 -2.01815441e-02\n",
      "  -1.12658013e-02  4.00341973e-02 -6.25530607e-04 -1.70896828e-01]]\n"
     ]
    }
   ],
   "source": [
    "model = SentenceTransformer('all-MiniLM-L6-v2')\n",
    "embedding = model.encode([\"I love jazz\"])\n",
    "print(embedding)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "8f50259a",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Found cached dataset rotten_tomatoes (/home/zhenya/.cache/huggingface/datasets/rotten_tomatoes/default/1.0.0/40d411e45a6ce3484deed7cc15b82a53dad9a72aafd9f86f8f227134bec5ca46)\n",
      "Found cached dataset rotten_tomatoes (/home/zhenya/.cache/huggingface/datasets/rotten_tomatoes/default/1.0.0/40d411e45a6ce3484deed7cc15b82a53dad9a72aafd9f86f8f227134bec5ca46)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BERT embeddings: 11.410213232040405 s\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.77      0.79      0.78       160\n",
      "           1       0.79      0.76      0.77       160\n",
      "\n",
      "    accuracy                           0.78       320\n",
      "   macro avg       0.78      0.78      0.78       320\n",
      "weighted avg       0.78      0.78      0.78       320\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "vectorize = lambda x: get_sentence_vector(x, model)\n",
    "(train_df, test_df) = load_train_test_dataset_pd()\n",
    "start = time.time()\n",
    "(X_train, X_test, y_train, y_test) = create_train_test_data(train_df, test_df, vectorize)\n",
    "print(f\"BERT embeddings: {time.time() - start} s\")\n",
    "clf = train_classifier(X_train, y_train)\n",
    "test_classifier(test_df, clf)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8ed1eba8",
   "metadata": {},
   "source": [
    "# OpenAI embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "547aae34",
   "metadata": {},
   "outputs": [],
   "source": [
    "import openai\n",
    "openai.api_key = OPEN_AI_KEY"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f48c8378",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = \"text-embedding-ada-002\"\n",
    "text = \"I love jazz\"\n",
    "response = openai.Embedding.create(\n",
    "    input=text,\n",
    "    model=model\n",
    ")\n",
    "embeddings = response['data'][0]['embedding']\n",
    "print(embeddings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d275c514",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_sentence_vector(text, model):\n",
    "    text = \"I love jazz\"\n",
    "    response = openai.Embedding.create(\n",
    "        input=text,\n",
    "        model=model\n",
    "    )\n",
    "    embeddings = response['data'][0]['embedding']\n",
    "    return embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0bb68a04",
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "vectorize = lambda x: get_sentence_vector(x, model)\n",
    "(train_df, test_df) = load_train_test_dataset_pd()\n",
    "start = time.time()\n",
    "(X_train, X_test, y_train, y_test) = create_train_test_data(train_df, test_df, vectorize)\n",
    "print(f\"OpenAI embeddings: {time.time() - start} s\")\n",
    "clf = train_classifier(X_train, y_train)\n",
    "test_classifier(test_df, clf)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
